{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 分类问题"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEWCAYAAABliCz2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAcAElEQVR4nO3de5wU5Z3v8c8XASPgoggiFy9BRcEAw8UVXknMcLKgQAy6ujm6XuIthDVGjYYcN+tx3c0FNbo5ibmgRhDFmIsmBhNDNCQjrolRNDhAyHiByYEogokTAVEBf/tHFdCDAzTQPd3D832/Xv2a6qe6qn710Hyn+qnqGkUEZma292tX6QLMzKx1OPDNzBLhwDczS4QD38wsEQ58M7NEOPDNzBLhwDfbhqRpkv5vpeswKzX5OnwrFUmNQE9gU0Fz/4h4aQ/WWQvMioi+e1Zd2yTpTmBFRFxT6Vqs7fMRvpXaKRHRpeCx22FfCpLaV3L7e0LSPpWuwfYuDnxrFZJGSvqNpCZJz+ZH7pvnXSBpiaQ1kpZK+mTe3hn4OdBb0tr80VvSnZK+WLB8raQVBc8bJf0fSfXAOknt8+Xul7Ra0jJJl+2g1i3r37xuSZ+TtErSy5JOlTRe0nOS/irp8wXLXifpPknfz/fnGUlDCuYPkFSX98NiSR/dZrvflvSQpHXARcDZwOfyfX8wf93Vkl7M1/8HSacVrON8Sf8t6SZJr+X7Oq5gfjdJMyS9lM9/oGDeRyQtyGv7jaTBRf8DW5vgwLeyk9QH+BnwRaAb8Fngfkk98pesAj4C/B1wAfBVScMiYh0wDnhpNz4xnAVMAA4A3gEeBJ4F+gAfBq6QdFKR6zoEeE++7LXA7cA5wHDgg8C1kvoVvH4i8MN8X78LPCCpg6QOeR0PAwcDnwbukXRMwbL/DHwJ2B+4C7gHuDHf91Py17yYb7cr8B/ALEm9CtZxAtAAdAduBO6QpHze3UAn4Li8hq8CSBoGTAc+CRwE3ArMlrRvkX1kbYAD30rtgfwIsang6PEc4KGIeCgi3omIR4D5wHiAiPhZRLwYmUfJAvGDe1jH1yNieUSsB44HekTEf0bE2xGxlCy0zyxyXRuAL0XEBuB7ZEH6tYhYExGLgcVA4dHw0xFxX/76/yL7ZTEyf3QBrs/r+BXwU7JfTpv9JCIez/vpzZaKiYgfRsRL+Wu+DzwP/H3BS/4UEbdHxCZgJtAL6Jn/UhgHTI6I1yJiQ97fAJ8Abo2I30XEpoiYCbyV12x7iTY7vmlV69SI+OU2bYcD/yTplIK2DsCvAfIhh38H+pMdhHQCFu5hHcu32X5vSU0FbfsAjxW5rr/k4QmwPv/5SsH89WRB/q5tR8Q7+XBT783zIuKdgtf+ieyTQ0t1t0jSecCVwBF5UxeyX0KbrSzY/hv5wX0Xsk8cf42I11pY7eHAxyV9uqCtY0Hdthdw4FtrWA7cHRGf2HZGPmRwP3Ae2dHthvyTweYhiJYuI1tH9kths0NaeE3hcsuBZRFx9O4UvxsO3TwhqR3QF9g8FHWopHYFoX8Y8FzBstvub7Pnkg4n+3TyYeC3EbFJ0gK29teOLAe6STogIppamPeliPhSEeuxNspDOtYaZgGnSDpJ0j6S3pOfDO1LdhS5L7Aa2Jgf7Y8tWPYV4CBJXQvaFgDj8xOQhwBX7GT7TwKv5ydy98treJ+k40u2h80Nl/SP+RVCV5ANjTwB/I7sl9Xn8jH9WuAUsmGi7XkFKDw/0Jnsl8BqyE54A+8rpqiIeJnsJPi3JB2Y13BiPvt2YLKkE5TpLGmCpP2L3GdrAxz4VnYRsZzsRObnyYJqOTAFaBcRa4DLgB8Ar5GdtJxdsOwfgXuBpfl5gd5kJx6fBRrJxvu/v5PtbyIL1hpgGfAq8B2yk57l8BPgf5Ptz7nAP+bj5W8DHyUbR38V+BZwXr6P23MHMHDzOZGI+ANwM/Bbsl8Gg4DHd6G2c8nOSfyR7GT5FQARMZ9sHP8bed0vAOfvwnqtDfAXr8xKSNJ1wFERcU6lazHblo/wzcwS4cA3M0uEh3TMzBLhI3wzs0RU7XX4BxxwQBx11FGVLqNqrFu3js6dO1e6jKrgvtjKfdGc+wOefvrpVyOiR0vzqjbwe/bsyfz58ytdRtWoq6ujtra20mVUBffFVu6L5twfIOlP25vnIR0zs0Q48M3MEuHANzNLhAPfzCwRDnwzs0Q48M3MEuHANzNLhAPfzCwRDnwzs0Q48M3MEuHANzNLhAPfzCwRDnwzs0Q48M3MEuHANzNLhAPfzCwRDnwzs0Q48M3MEuHANzNLhAPfzCwRDnwzs0Q48M3MEuHANzNLhAPfzCwRDnwzs0Q48M3MEuHANzNLhAPfzCwRDnwzs0Q48M3MEuHANzNLhAPfzCwRDnwzs0Q48M3MEuHANzNLhAPfzCwRDnwzs0QoIipdQ4sO63dUtPvY1ypdRtW4atBGbl7YvtJlVAX3xVbui+aqqT8ar59Qke1KejoiRrQ0z0f4ZmZltHz5ckaPHs2AAQM47rjj+NrXsgPZKVOmcOyxxzJ48GBOO+00mpqatixTX1/PqFGjOO644xg0aBBvvvlmSWopW+BLukzSEkkhqT5//EbSkHJt08ys2rRv356bb76ZJUuW8MQTT/DNb36TP/zhD4wZM4ZFixZRX19P//79mTp1KgAbN27knHPOYdq0aSxevJi6ujo6dOhQmlpKspaWXQKMA3oBSyLiNUnjgNuAE8q4XTOzqtGrVy969eoFwP7778+AAQP485//zNixY7e8ZuTIkdx3330APPzwwwwePJghQ7Jj44MOOqhktZTlCF/SNKAfMBs4ISJey2c9AfQtxzbNzKpdY2Mjv//97znhhObHvNOnT2fcuHEAPPfcc0jipJNOYtiwYdx4440l235ZjvAjYrKkk4HREfFqwayLgJ9vbzlJk4BJAN279+DaQRvLUV6b1HO/7ISUuS8KuS+aq6b+qKura/Z8/fr1XH755Vx88cU888wzW9pnzZpFU1MTffr0oa6ujoaGBn75y18ybdo09t13X6666ir22Wcfhg8fvsc1tdrpbEmjyQL/A9t7TUTcRjbkw2H9jopqOdteDarp6oNKc19s5b5orpr6o/Hs2i3TGzZs4CMf+QiTJ0/myiuv3NI+c+ZMFi9ezNy5c+nUqRMAK1euZP369UycOBGAp556infeeYfa2lr2VKtcpSNpMPAdYGJE/KU1tmlmVg0igosuuogBAwY0C/s5c+Zwww03MHv27C1hD3DSSSdRX1/PG2+8wcaNG3n00UcZOHBgSWop+69CSYcBPwLOjYjnyr09M7Nq8vjjj3P33XczaNAgampqAPjyl7/MZZddxltvvcWYMWOA7MTttGnTOPDAA7nyyis5/vjjkcT48eOZMKE01/SX7YtXkhqBEcD1wOnAn/JZG7f3pYBCxxxzTDQ0NJSltraorq6uJB/p9gbui63cF825P3b8xauyHeFHxBH55MX5w8zMKsjftDUzS4QD38wsEQ58M7NEOPDNzBLhwDczS4QD38wsEQ58M7NEOPDNzBLhwDczS4QD38wsEQ58M7NEOPDNzBLhwDczS4QD38wsEQ58M7NEOPDNzBLhwDczS4QD38wsEQ58M7NEOPDNzBLhwDczS4QD38wsEQ58M7NEOPDNzBLhwDczS4QD38wsEQ58M7NEOPDNzBLhwDczS4QD38wsEQ58M7NEOPDNzBLhwDczS0T7ShewPes3bOKIq39W6TKqxlWDNnK++wOonr5ovH4CABdeeCE//elPOfjgg1m0aBEAU6ZM4cEHH6Rjx44ceeSRzJgxgwMOOIDGxkYGDBjAMcccA8DIkSOZNm1axfbB0lLWI3xJl0laIukeSV+X9IKkeknDyrlds9Z0/vnnM2fOnGZtY8aMYdGiRdTX19O/f3+mTp26Zd6RRx7JggULWLBggcPeWlW5h3QuAcYD9wBH549JwLfLvF2zVnPiiSfSrVu3Zm1jx46lffvsA/TIkSNZsWJFJUoza6ZsgS9pGtAPmA38GLgrMk8AB0jqVa5tm1WT6dOnM27cuC3Ply1bxtChQ/nQhz7EY489VsHKLDVlG8OPiMmSTgZGA3cCywtmrwD6AC8XLiNpEtknALp378G1gzaWq7w2p+d+2di1VU9f1NXVbZleuXIl69ata9YGMGvWLJqamujTpw91dXW8/fbbfPe736Vr1640NDRw+umnM2PGDDp37rxbNaxdu/Zd20yZ+2PHWuukrVpoi3c1RNwG3AZwWL+j4uaFVXtOudVdNWgj7o9MtfRF49m1W6cbG+ncuTO1tVvbZs6cyeLFi5k7dy6dOnV61/K1tbXce++99OzZkxEjRuxWDXV1dc22mTr3x4611mWZK4BDC573BV5qpW2btbo5c+Zwww03MHv27GZhv3r1ajZt2gTA0qVLef755+nXr1+lyrTE7HLgSzpQ0uBdXGw2cJ4yI4G/RcTLO1vIrC0466yzGDVqFA0NDfTt25c77riDSy+9lDVr1jBmzBhqamqYPHkyAPPmzWPw4MEMGTKEM844g2nTpr3rhK9ZuRT1uVhSHfDR/PULgNWSHo2IK4vczkNkV+u8ALwBXLDrpZpVp3vvvfddbRdddFGLrz399NM5/fTTy12SWYuKHQjtGhGvS7oYmBER/y6pfmcLRcQRBU8/tSuF7ddhHxryL7ZYNjZZOGacMveF2e4pdkinfX4Z5ceAn5axHjMzK5NiA/8/gV8AL0bEU5L6Ac+XrywzMyu1ooZ0IuKHwA8Lni8FPBBpZtaGFHWEL6m/pLmSFuXPB0u6prylmZlZKRU7pHM78K/ABoCIqAfOLFdRZmZWesUGfqeIeHKbtsp/t93MzIpWbOC/KulI8tshSDqDbe6DY2Zm1a3Y6/A/RXaPm2Ml/RlYBpxdtqrMzKzkdhr4ktoBIyLiHyR1BtpFxJryl2ZmZqW00yGdiHgHuDSfXuewNzNrm4odw39E0mclHSqp2+ZHWSszM7OSKnYM/8L8Z+H9cILsL1qZmVkbUOw3bd9b7kLMzKy8ir098nkttUfEXaUtx8zMyqXYIZ3jC6bfA3wYeAZw4JuZtRHFDul8uvC5pK7A3WWpyMzMymJ3/6btG8DRpSzEzMzKq9gx/AfJb6tA9ktiIAW3SzYzs+pX7Bj+TQXTG4E/RcSKMtRjZmZlUuyQzviIeDR/PB4RKyTdUNbKzMyspIoN/DEttI0rZSFmZlZeOxzSkfQvwCVAP0n1BbP2Bx4vZ2FmZlZaOxvD/y7wc2AqcHVB+5qI+GvZqjIzs5LbYeBHxN+AvwFnAUg6mOyLV10kdYmI/1/+Es3MrBSK/SPmp0h6nuwPnzwKNJId+ZuZWRtR7EnbLwIjgefyG6l9GI/hm5m1KcUG/oaI+AvQTlK7iPg1UFPGuszMrMSK/eJVk6QuwGPAPZJWkX0By8zM2ohij/Ankt0/5wpgDvAicEq5ijIzs9Ir9m6Z6yQdDhwdETMldQL2KW9pZmZWSsVepfMJ4D7g1rypD/BAuYoyM7PSK3ZI51PA+4HXASLieeDgchVlZmalV2zgvxURb29+Iqk9W2+XbGZmbUCxV+k8KunzwH6SxpDdX+fB8pUF6zds4oirf1bOTbQpVw3ayPnuD6ByfdF4/YRW36ZZKRV7hH81sBpYCHwSeAi4plxFmVWzCy+8kIMPPpj3ve99W9qmTJnCsccey+DBgznttNNoamoC4JFHHmH48OEMGjSI4cOH86tf/apSZZvtOPAlHQYQEe9ExO0R8U8RcUY+vdMhHUmXSVoi6TVJ9ZIWSJov6QOl2gGz1nb++eczZ86cZm1jxoxh0aJF1NfX079/f6ZOnQpA9+7defDBB1m4cCEzZ87k3HPPrUTJZsDOj/C3XIkj6f7dWP8lwHjgUGBIRNQAFwLf2Y11mVWFE088kW7dujVrGzt2LO3bZyOkI0eOZMWK7A/CDR06lN69ewNw3HHH8eabb/LWW2+1bsFmuZ0Fvgqm++3KiiVNy5eZDXyi4BNBZ3zC1/Zi06dPZ9y4d/99oPvvv5+hQ4ey7777VqAqs52ftI3tTO9UREyWdDIwOiJelXQa2X31DwZaPPslaRIwCaB79x5cO8h3b9is537ZyUqrXF/U1dVtmV65ciXr1q1r1gYwa9Ysmpqa6NOnT7N5y5Yt45prruHGG2981zJ7Yu3atSVdX1vn/tixnQX+EEmvkx3p75dPkz+PiPi7YjcUET8GfizpROALwD+08JrbgNsADut3VNy8sNiLiPZ+Vw3aiPsjU6m+aDy7dut0YyOdO3emtnZr28yZM1m8eDFz586lU6dOW9pXrFjBpEmT+MEPfsD73//+ktZUV1fXrIbUuT92bGd/AKXkt0+IiHmSjpTUPSJeLfX6zSphzpw53HDDDTz66KPNwr6pqYkJEyYwderUkoe92a4q9rLMPSLpKEnKp4cBHYG/tMa2zUrtrLPOYtSoUTQ0NNC3b1/uuOMOLr30UtasWcOYMWOoqalh8uTJAHzjG9/ghRde4Atf+AI1NTXU1NSwatWqCu+BpUpFXF25+yuXGoERwEXAecAGYD0wJSL+e0fLHnPMMdHQ0FC22toaf1Tdyn2xlfuiOfcHSHo6Ika0NK+sA6ERcUQ+eUP+MDOzCmmVIR0zM6s8B76ZWSIc+GZmiXDgm5klwoFvZpYIB76ZWSIc+GZmiXDgm5klwoFvZpYIB76ZWSIc+GZmiXDgm5klwoFvZpYIB76ZWSIc+GZmiXDgm5klwoFvZpYIB76ZWSIc+GZmiXDgm5klwoFvZpYIB76ZWSIc+GZmiXDgm5klwoFvZpYIB76ZWSIc+GZmiXDgm5klwoFvZpYIB76ZWSIc+GZmiXDgm5klwoFvZpaI9pUuYHvWb9jEEVf/rNJlVI2rBm3k/BL0R+P1E7ZMNzU1cfHFF7No0SIkMX36dEaNGgXATTfdxJQpU1i9ejXdu3ff4+2aWeWV7Qhf0mWSlki6X9JvJb0l6bPl2p7tussvv5yTTz6ZP/7xjzz77LMMGDAAgOXLl/PII49w2GGHVbhCMyulch7hXwKMA9YBhwOnlnFbtotef/115s2bx5133glAx44d6dixIwCf+cxnuPHGG5k4cWIFKzSzUivLEb6kaUA/YDZwdkQ8BWwox7Zs9yxdupQePXpwwQUXMHToUC6++GLWrVvH7Nmz6dOnD0OGDKl0iWZWYoqI8qxYagRGRMSr+fPrgLURcdMOlpkETALo3r3H8Gv/3+1lqa0t6rkfvLJ+z9czqE9XABoaGrjkkku45ZZbGDhwILfccgsdOnTg2Wef5Stf+QpdunThzDPP5NZbb6Vr1657vuESWrt2LV26dKl0GVXBfdGc+wNGjx79dESMaGleVQV+ocP6HRXtPva1stTWFl01aCM3L9zzEbjNJ21XrlzJyJEjaWxsBOCxxx7juuuuY+HChXTq1AmAFStW0Lt3b5588kkOOeSQPd52qdTV1VFbW1vpMqqC+6I59wdI2m7g+7LMRB1yyCEceuihNDQ0ADB37lyGDRvGqlWraGxspLGxkb59+/LMM89UVdib2e6r2ssyrfxuueUWzj77bN5++2369evHjBkzKl2SmZVR2QNf0iHAfODvgHckXQEMjIjXy71t27Gamhrmz5+/3fmbh3vMbO9QtsCPiCMKnvbd1eX367APDQVfEkpdXV0djWfXVroMM2vDPIZvZpYIB76ZWSIc+GZmiXDgm5klwoFvZpYIB76ZWSIc+GZmiXDgm5klwoFvZpYIB76ZWSIc+GZmiXDgm5klwoFvZpYIB76ZWSIc+GZmiXDgm5klwoFvZpYIB76ZWSIc+GZmiXDgm5klwoFvZpYIB76ZWSIc+GZmiXDgm5klwoFvZpYIB76ZWSIc+GZmiXDgm5klwoFvZpYIB76ZWSIc+GZmiXDgm5klwoFvZpYIB76ZWSIc+GZmiXDgm5klwoFvZpYIB76ZWSIUEZWuoUWS1gANla6jinQHXq10EVXCfbGV+6I59wccHhE9WprRvrUr2QUNETGi0kVUC0nz3R8Z98VW7ovm3B875iEdM7NEOPDNzBJRzYF/W6ULqDLuj63cF1u5L5pzf+xA1Z60NTOz0qrmI3wzMyshB76ZWSKqMvAlnSypQdILkq6udD2tTVKjpIWSFkian7d1k/SIpOfznwdWus5ykTRd0ipJiwraWtx/Zb6ev1fqJQ2rXOWlt52+uE7Sn/P3xwJJ4wvm/WveFw2STqpM1eUh6VBJv5a0RNJiSZfn7Um+N3ZH1QW+pH2AbwLjgIHAWZIGVraqihgdETUF1xRfDcyNiKOBufnzvdWdwMnbtG1v/8cBR+ePScC3W6nG1nIn7+4LgK/m74+aiHgIIP9/ciZwXL7Mt/L/T3uLjcBVETEAGAl8Kt/nVN8bu6zqAh/4e+CFiFgaEW8D3wMmVrimajARmJlPzwROrWAtZRUR84C/btO8vf2fCNwVmSeAAyT1ap1Ky287fbE9E4HvRcRbEbEMeIHs/9NeISJejohn8uk1wBKgD4m+N3ZHNQZ+H2B5wfMVeVtKAnhY0tOSJuVtPSPiZcje+MDBFauuMra3/6m+Xy7NhymmFwzvJdMXko4AhgK/w++NolVj4KuFttSuHX1/RAwj+0j6KUknVrqgKpbi++XbwJFADfAycHPenkRfSOoC3A9cERGv7+ilLbTtdf2xK6ox8FcAhxY87wu8VKFaKiIiXsp/rgJ+TPax/JXNH0fzn6sqV2FFbG//k3u/RMQrEbEpIt4BbmfrsM1e3xeSOpCF/T0R8aO82e+NIlVj4D8FHC3pvZI6kp2Eml3hmlqNpM6S9t88DYwFFpH1wcfzl30c+EllKqyY7e3/bOC8/IqMkcDfNn+831ttMw59Gtn7A7K+OFPSvpLeS3ay8snWrq9cJAm4A1gSEf9VMMvvjWJFRNU9gPHAc8CLwL9Vup5W3vd+wLP5Y/Hm/QcOIrsC4fn8Z7dK11rGPriXbKhiA9lR2kXb23+yj+3fzN8rC4ERla6/Ffri7nxf68lCrVfB6/8t74sGYFyl6y9xX3yAbEimHliQP8an+t7YnYdvrWBmlohqHNIxM7MycOCbmSXCgW9mlggHvplZIhz4ZmaJqOY/Ym5WFpI2kV2mt9mpEdFYoXLMWo0vy7TkSFobEV1acXvtI2Jja23PbHs8pGO2DUm9JM3L7zW/SNIH8/aTJT0j6VlJc/O2bpIeyG9k9oSkwXn7dZJuk/QwcJekfSR9RdJT+Ws/WcFdtER5SMdStJ+kBfn0sog4bZv5/wz8IiK+lN9PvpOkHmT3rTkxIpZJ6pa/9j+A30fEqZL+F3AX2U3NAIYDH4iI9fldT/8WEcdL2hd4XNLDkd3G2KxVOPAtResjomYH858Cpuc36nogIhZIqgXmbQ7oiNh8j/oPAKfnbb+SdJCkrvm82RGxPp8eCwyWdEb+vCvZvW4c+NZqHPhm24iIefktqScAd0v6CtBEy7fW3dEteNdt87pPR8QvSlqs2S7wGL7ZNiQdDqyKiNvJ7s44DPgt8KH8LpQUDOnMA87O22qBV6Ple7T/AviX/FMDkvrnd0M1azU+wjd7t1pgiqQNwFrgvIhYnY/D/0hSO7J7ro8BrgNmSKoH3mDrbXq39R3gCOCZ/Da/q9mL/0ylVSdflmlmlggP6ZiZJcKBb2aWCAe+mVkiHPhmZolw4JuZJcKBb2aWCAe+mVki/gdZsW+5PWp9jQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9473684210526315\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"740pt\" height=\"305pt\"\n",
       " viewBox=\"0.00 0.00 740.49 305.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 301)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-301 736.4856,-301 736.4856,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"254.3431\" cy=\"-279\" rx=\"66.8882\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"254.3431\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">f2&lt;2.45000005</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"208.3431\" cy=\"-192\" rx=\"85.2851\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"208.3431\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">leaf=&#45;0.0696000084</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;1 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"#0000ff\" d=\"M244.8119,-260.9735C238.3974,-248.8418 229.825,-232.6287 222.6031,-218.9698\"/>\n",
       "<polygon fill=\"#0000ff\" stroke=\"#0000ff\" points=\"225.6305,-217.2077 217.8622,-210.0034 219.4423,-220.4797 225.6305,-217.2077\"/>\n",
       "<text text-anchor=\"middle\" x=\"269.3431\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">yes, missing</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>2</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"374.3431\" cy=\"-192\" rx=\"62.2891\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"374.3431\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">f3&lt;1.6500001</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;2 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>0&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"#ff0000\" d=\"M280.1398,-262.3307C289.1554,-256.3663 299.2837,-249.508 308.3431,-243 320.5513,-234.2301 333.7106,-224.187 345.0425,-215.3451\"/>\n",
       "<polygon fill=\"#ff0000\" stroke=\"#ff0000\" points=\"347.225,-218.0815 352.9312,-209.1545 342.9035,-212.5746 347.225,-218.0815\"/>\n",
       "<text text-anchor=\"middle\" x=\"334.8431\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">no</text>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>3</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"264.3431\" cy=\"-105\" rx=\"27.8951\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"264.3431\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">f2&lt;5</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;3 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>2&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"#0000ff\" d=\"M352.8729,-175.019C334.8998,-160.8039 309.202,-140.4793 290.1191,-125.3864\"/>\n",
       "<polygon fill=\"#0000ff\" stroke=\"#0000ff\" points=\"292.1428,-122.5247 282.1283,-119.0664 287.8005,-128.015 292.1428,-122.5247\"/>\n",
       "<text text-anchor=\"middle\" x=\"361.3431\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">yes, missing</text>\n",
       "</g>\n",
       "<!-- 4 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>4</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"459.3431\" cy=\"-105\" rx=\"66.8882\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"459.3431\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">f2&lt;4.85000038</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;4 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>2&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"#ff0000\" d=\"M391.5442,-174.3943C403.987,-161.6586 420.9788,-144.2671 434.8387,-130.081\"/>\n",
       "<polygon fill=\"#ff0000\" stroke=\"#ff0000\" points=\"437.6828,-132.1783 442.1677,-122.5796 432.6758,-127.2865 437.6828,-132.1783\"/>\n",
       "<text text-anchor=\"middle\" x=\"429.8431\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">no</text>\n",
       "</g>\n",
       "<!-- 5 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>5</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"77.3431\" cy=\"-18\" rx=\"77.1866\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"77.3431\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">leaf=0.139534876</text>\n",
       "</g>\n",
       "<!-- 3&#45;&gt;5 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>3&#45;&gt;5</title>\n",
       "<path fill=\"none\" stroke=\"#0000ff\" d=\"M241.4133,-94.3321C211.6535,-80.4866 159.09,-56.032 121.4743,-38.5316\"/>\n",
       "<polygon fill=\"#0000ff\" stroke=\"#0000ff\" points=\"122.7992,-35.2878 112.256,-34.2429 119.8463,-41.6345 122.7992,-35.2878\"/>\n",
       "<text text-anchor=\"middle\" x=\"217.3431\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">yes, missing</text>\n",
       "</g>\n",
       "<!-- 6 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>6</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"264.3431\" cy=\"-18\" rx=\"91.784\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"264.3431\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">leaf=&#45;2.55448485e&#45;09</text>\n",
       "</g>\n",
       "<!-- 3&#45;&gt;6 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>3&#45;&gt;6</title>\n",
       "<path fill=\"none\" stroke=\"#ff0000\" d=\"M264.3431,-86.9735C264.3431,-75.1918 264.3431,-59.5607 264.3431,-46.1581\"/>\n",
       "<polygon fill=\"#ff0000\" stroke=\"#ff0000\" points=\"267.8432,-46.0033 264.3431,-36.0034 260.8432,-46.0034 267.8432,-46.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"271.8431\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">no</text>\n",
       "</g>\n",
       "<!-- 7 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>7</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"459.3431\" cy=\"-18\" rx=\"85.2851\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"459.3431\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">leaf=&#45;0.0120000029</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;7 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>4&#45;&gt;7</title>\n",
       "<path fill=\"none\" stroke=\"#0000ff\" d=\"M459.3431,-86.9735C459.3431,-75.1918 459.3431,-59.5607 459.3431,-46.1581\"/>\n",
       "<polygon fill=\"#0000ff\" stroke=\"#0000ff\" points=\"462.8432,-46.0033 459.3431,-36.0034 455.8432,-46.0034 462.8432,-46.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"494.3431\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">yes, missing</text>\n",
       "</g>\n",
       "<!-- 8 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>8</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"647.3431\" cy=\"-18\" rx=\"85.2851\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"647.3431\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">leaf=&#45;0.0685714409</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;8 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>4&#45;&gt;8</title>\n",
       "<path fill=\"none\" stroke=\"#ff0000\" d=\"M493.3885,-89.245C524.0488,-75.0564 569.3144,-54.1091 602.8341,-38.5973\"/>\n",
       "<polygon fill=\"#ff0000\" stroke=\"#ff0000\" points=\"604.3496,-41.7526 611.955,-34.3764 601.4097,-35.3999 604.3496,-41.7526\"/>\n",
       "<text text-anchor=\"middle\" x=\"571.8431\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">no</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x1a22f76f98>"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "from xgboost import XGBClassifier\n",
    "from xgboost import plot_importance\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "\n",
    "X,y = load_iris(return_X_y=True) # iris 花\n",
    "x_train, x_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "\n",
    "model = XGBClassifier(learning_rate=0.1,\n",
    "                      n_estimators=100,          # 树的个数--100棵树建立xgboost\n",
    "                      max_depth=6,               # 树的深度\n",
    "                      min_child_weight = 1,      # 叶子节点最小权重\n",
    "                      gamma=0.,                  # 惩罚项中叶子结点个数前的参数\n",
    "                      subsample=0.8,             # 随机选择80%样本建立决策树\n",
    "                      colsample_btree=0.8,       # 随机选择80%特征建立决策树\n",
    "                      objective='multi:softmax', # 指定损失函数\n",
    "                      scale_pos_weight=1         # 解决样本个数不平衡的问题\n",
    "                      )\n",
    "\n",
    "model.fit(x_train,y_train)\n",
    "\n",
    "# plot feature importance\n",
    "plot_importance(model)                                                            \n",
    "plt.show()\n",
    "\n",
    "# make prediction for test data\n",
    "y_pred = model.predict(x_test)\n",
    "\n",
    "# model evaluate\n",
    "accuracy = accuracy_score(y_test,y_pred)\n",
    "print(accuracy)\n",
    "\n",
    "# 绘制决策树\n",
    "xgb.to_graphviz(model, num_trees=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "XGBoost 参数一般分为三种：\n",
    "\n",
    "+ General parameters：参数控制在提升（boosting）过程中使用哪种booster，常用的booster有树模型（tree）和线性模型（linear model）。\n",
    "+ Booster parameters：这取决于使用哪种booster。\n",
    "+ Learning Task parameters：控制学习的场景，例如在回归问题中会使用不同的参数控制排序\n",
    "\n",
    "### 通用参数\n",
    "\n",
    "+ booster\n",
    "\n",
    "gbtree和gblinear \n",
    "\n",
    "+ silent\n",
    "\n",
    "默认 0，为1时模型运行不输出 \n",
    "\n",
    "+ nthread\n",
    "\n",
    "默认值为最大可能的线程数\n",
    "\n",
    "\n",
    "\n",
    "### Booster参数\n",
    "\n",
    "+ n_estimator\n",
    "\n",
    "生成的最大树的数目，也是最大的迭代次数\n",
    "\n",
    "+ learning_rate\n",
    "\n",
    "每一步迭代的步长 \n",
    "\n",
    "+ gamma\n",
    "\n",
    "默认 0，Gamma指定了节点分裂所需的最小损失函数下降值\n",
    "\n",
    "+ subsample\n",
    "\n",
    "默认 1，这个参数控制对于每棵树，随机采样的比例 \n",
    "\n",
    "+ colsample_bytree\n",
    "\n",
    "默认 1，用来控制每棵随机采样的列数的占比\n",
    "\n",
    "+ colsample_bylevel\n",
    "\n",
    "默认 1，每棵树每次节点分裂的时候列采样的比例\n",
    "\n",
    "+ max_depth\n",
    "\n",
    "默认 6，我们常用3-10之间的数字。这个值为树的最大深度。这个值是用来控制过拟合的。max_depth越大，模型学习的更加具体\n",
    "\n",
    "+ max_delta_step\n",
    "\n",
    "默认 0，这个参数限制了每棵树权重改变的最大步长，如果这个参数的值为0,则意味着没有约束\n",
    "\n",
    "+ lambda\n",
    "\n",
    "默认 0，权重的L2正则化项\n",
    "\n",
    "+ alpha\n",
    "\n",
    "默认 0，权重的L1正则化项\n",
    "\n",
    "+ scale_pos_weight\n",
    "\n",
    "默认 1，在各类别样本十分不平衡时，把这个参数设定为一个正值，可以使算法更快收敛。通常可以将其设置为负样本的数目与正样本数目的比值\n",
    "\n",
    "\n",
    "### 学习目标参数\n",
    "+ objective\n",
    "\n",
    "reg:linear– 线性回归\n",
    "\n",
    "reg:logistic – 逻辑回归\n",
    "\n",
    "binary:logistic – 二分类逻辑回归，输出为概率\n",
    "\n",
    "binary:logitraw – 二分类逻辑回归，输出的结果为wTx\n",
    "\n",
    "count:poisson – 计数问题的poisson回归，输出结果为poisson分布。在poisson回归中，max_delta_step的缺省值为0.7 (used to safeguard optimization)\n",
    "\n",
    "multi:softmax – 设置 XGBoost 使用softmax目标函数做多分类，需要设置参数num_class（类别个数）\n",
    "\n",
    "multi:softprob – 如同softmax，但是输出结果为ndata*nclass的向量，其中的值是每个数据分为每个类的概率\n",
    "\n",
    "+ eval_metric\n",
    "\n",
    "rmse: 均方根误差\n",
    "\n",
    "mae: 平均绝对值误差\n",
    "\n",
    "logloss: negative log-likelihood\n",
    "\n",
    "error: 二分类错误率。其值通过错误分类数目与全部分类数目比值得到。对于预测，预测值大于0.5被认为是正类，其它归为负类。 error@t: 不同的划分阈值可以通过 ‘t’进行设置\n",
    "\n",
    "merror: 多分类错误率，计算公式为(wrong cases)/(all cases)\n",
    "\n",
    "mlogloss: 多分类log损失\n",
    "\n",
    "auc: 曲线下的面积\n",
    "\n",
    "ndcg: Normalized Discounted Cumulative Gain\n",
    "\n",
    "map: 平均正确率\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " `scoring`参数选择 : https://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter\n",
    "\n",
    "![](https://tva1.sinaimg.cn/large/007S8ZIlly1gflclt71k4j30o40oo7ak.jpg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 调参函数\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "def modify_params(parameters):\n",
    "    #param_grid = {param : param_list}\n",
    "    kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=7)\n",
    "    model = XGBClassifier()\n",
    "    grid_search = GridSearchCV(model, parameters, scoring=\"accuracy\", n_jobs=-1, cv=kfold)\n",
    "    grid_result = grid_search.fit(x_train,y_train)\n",
    "    means = grid_result.cv_results_['mean_test_score']\n",
    "    stds = grid_result.cv_results_['std_test_score']\n",
    "    params = grid_result.cv_results_['params']\n",
    "    for mean, stdev, param in zip(means, stds, params):\n",
    "        print(\"mean:%f ,std:%f with: %r\" % (mean, stdev, param))\n",
    "    print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean:0.955357 ,std:0.060176 with: {'learning_rate': 0.0001}\n",
      "mean:0.955357 ,std:0.060176 with: {'learning_rate': 0.001}\n",
      "mean:0.955357 ,std:0.060176 with: {'learning_rate': 0.01}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.1}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.2}\n",
      "mean:0.955357 ,std:0.060176 with: {'learning_rate': 0.3}\n",
      "Best: 0.964286 using {'learning_rate': 0.1}\n"
     ]
    }
   ],
   "source": [
    "# 选择学习率\n",
    "learning_rate_param={\"learning_rate\" : [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3]}\n",
    "modify_params(learning_rate_param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean:0.955357 ,std:0.060176 with: {'n_estimators': 10}\n",
      "mean:0.964286 ,std:0.043630 with: {'n_estimators': 100}\n",
      "mean:0.964286 ,std:0.043630 with: {'n_estimators': 500}\n",
      "mean:0.964286 ,std:0.043630 with: {'n_estimators': 1000}\n",
      "Best: 0.964286 using {'n_estimators': 100}\n"
     ]
    }
   ],
   "source": [
    "# 选择树的个数\n",
    "n_estimators_param={\"n_estimators\" : [10, 100, 500, 1000]}\n",
    "modify_params(n_estimators_param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean:0.955357 ,std:0.060176 with: {'learning_rate': 0.01, 'n_estimators': 50}\n",
      "mean:0.955357 ,std:0.060176 with: {'learning_rate': 0.01, 'n_estimators': 100}\n",
      "mean:0.973214 ,std:0.041444 with: {'learning_rate': 0.01, 'n_estimators': 200}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.01, 'n_estimators': 300}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.01, 'n_estimators': 500}\n",
      "mean:0.955357 ,std:0.060176 with: {'learning_rate': 0.02, 'n_estimators': 50}\n",
      "mean:0.973214 ,std:0.041444 with: {'learning_rate': 0.02, 'n_estimators': 100}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.02, 'n_estimators': 200}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.02, 'n_estimators': 300}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.02, 'n_estimators': 500}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.05, 'n_estimators': 50}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.05, 'n_estimators': 100}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.05, 'n_estimators': 200}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.05, 'n_estimators': 300}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.05, 'n_estimators': 500}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.1, 'n_estimators': 50}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.1, 'n_estimators': 100}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.1, 'n_estimators': 200}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.1, 'n_estimators': 300}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.1, 'n_estimators': 500}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.15, 'n_estimators': 50}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.15, 'n_estimators': 100}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.15, 'n_estimators': 200}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.15, 'n_estimators': 300}\n",
      "mean:0.964286 ,std:0.043630 with: {'learning_rate': 0.15, 'n_estimators': 500}\n",
      "Best: 0.973214 using {'learning_rate': 0.01, 'n_estimators': 200}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hz/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:813: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "parameters = {\n",
    "    'learning_rate': [0.01, 0.02, 0.05, 0.1, 0.15],\n",
    "    'n_estimators': [50, 100, 200, 300, 500]\n",
    "}\n",
    "modify_params(parameters)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 回归问题"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6.128512775416618\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXzU9bX/8ddhEVkURcSyKIjInhCBClwQw/WirCrWHxXlYhFui1pFwa311qKlVkVUqlQFC7iiVVC4Fq0oRC1KEWQHWdQUUCtL2QKokJzfH/NNmAkBBkjmm/B9Px+PeWS+23zPHGXOfJf5HHN3REQkusqFHYCIiIRLhUBEJOJUCEREIk6FQEQk4lQIREQiToVARCTiVAhEkmRmT5nZb8KOQ6S4mX5HICXNzLKBM4DcuNmN3f3rY3jNTOAFd693bNGVTWY2Cdjg7v8bdixS9umIQFKlt7tXi3scdREoDmZWIcz9HwszKx92DHJ8USGQUJlZezP7yMy2mdni4Jt+/rKBZrbSzHaa2Rdm9otgflXgLaCOmeUEjzpmNsnMRsZtn2lmG+Kms83sTjNbAuwyswrBdlPMbJOZfWlmNx8i1oLXz39tM7vDzDaa2TdmdrmZ9TCz1Wb2bzP7ddy2I8zsNTN7JXg/n5pZq7jlzcwsK8jDcjO7tNB+nzSzGWa2CxgEXAPcEbz3/wvWu8vMPg9ef4WZ9Yl7jZ+Z2d/N7GEz2xq81+5xy2uY2UQz+zpY/kbcsl5mtiiI7SMzS0/6P7CUCSoEEhozqwv8FRgJ1ABuA6aY2enBKhuBXsDJwEDgUTNr7e67gO7A10dxhNEP6AmcAuQB/wcsBuoCFwG3mNklSb7Wj4ATg23vAcYD/YE2wAXAPWbWMG79y4BXg/f6EvCGmVU0s4pBHO8AtYCbgBfNrEnctlcDvwdOAp4DXgQeCt5772Cdz4P9VgfuBV4ws9pxr9EOWAXUBB4C/mxmFix7HqgCtAhieBTAzFoDE4BfAKcBTwPTzaxSkjmSMkCFQFLljeAb5ba4b5v9gRnuPsPd89x9JjAf6AHg7n9198895n1iH5QXHGMcf3T39e6+B/gxcLq73+fuP7j7F8Q+zK9K8rX2Ar93973Ay8Q+YMe4+053Xw4sB+K/PS9w99eC9R8hVkTaB49qwANBHLOAN4kVrXzT3H1OkKfvigrG3V9196+DdV4B1gDnx63yT3cf7+65wLNAbeCMoFh0B4a4+1Z33xvkG+B/gKfd/R/unuvuzwLfBzHLcaLMnieVMudyd3+30Lz6wP8zs95x8yoCswGCUxe/BRoT+9JSBVh6jHGsL7T/Oma2LW5eeeDDJF9rS/ChCrAn+Ptt3PI9xD7gD9i3u+cFp63q5C9z97y4df9J7EijqLiLZGYDgGFAg2BWNWLFKd+/4va/OzgYqEbsCOXf7r61iJetD1xrZjfFzTshLm45DqgQSJjWA8+7+/8UXhCcepgCDCD2bXhvcCSRfyqjqNvddhErFvl+VMQ68dutB75093OPJvijcGb+EzMrB9QD8k9pnWlm5eKKwVnA6rhtC7/fhGkzq0/saOYi4GN3zzWzRezP16GsB2qY2Snuvq2IZb93998n8TpSRunUkITpBaC3mV1iZuXN7MTgImw9Yt86KwGbgH3B0cHFcdt+C5xmZtXj5i0CegQXPn8E3HKY/c8DdgQXkCsHMbQ0sx8X2ztM1MbMrgjuWLqF2CmWucA/iBWxO4JrBplAb2Knmw7mWyD++kNVYsVhE8QutAMtkwnK3b8hdvH9T2Z2ahBD52DxeGCImbWzmKpm1tPMTkryPUsZoEIgoXH39cQuoP6a2AfYeuB2oJy77wRuBv4CbCV2sXR63LafAZOBL4LrDnWIXfBcDGQTu57wymH2n0vsAzcD+BLYDDxD7GJrSZgG/JTY+/lv4IrgfPwPwKXEztNvBv4EDAje48H8GWief83F3VcAo4GPiRWJNGDOEcT238SueXxG7CL9LQDuPp/YdYIngrjXAj87gteVMkA/KBNJATMbATRy9/5hxyJSmI4IREQiToVARCTidGpIRCTidEQgIhJxZfJ3BKeccoo3atQo7DBKjV27dlG1atWwwygVlItEykeiKOdjwYIFm9399KKWlclCcMYZZzB//vywwyg1srKyyMzMDDuMUkG5SKR8JIpyPszsnwdbplNDIiIRp0IgIhJxKgQiIhGnQiAiEnEqBCIiEadCICIScSoEIiIRp0IgIhJxKgQiIhGnQiAiEnEqBCIiEadCICISktzcXM477zx69eqVMP+mm26iWrVqBdPr1q2jS5cunHfeeaSnpzNjxoxijSOUQmBmN5vZSjN70cz+aGZrzWyJmbUOIx4RkTCMGTOGZs2aJcybP38+27ZtS5g3cuRI+vbty8KFC3n55Ze54YYbijWOsEYfvYFYo+5mwE3AuUA74Mng7yHt2ZtLg7v+WqIBliXD0/bxM+UDUC4KUz4SlYZ8ZD/QE4ANGzbw17/+lbvvvptHHnkEiB0h3H777bz00ku8/vrrBduYGTt27ABg+/bt1KlTp1hjSnkhMLOngIbAdKAx8DOPtUmba2anmFltd/8m1XGJiKTSLbfcwkMPPcTOnTsL5j3xxBNceuml1K5dO2HdESNGcPHFF/P444+za9cu3n333WKNJeWFwN2HmFk3oAswCVgft3gDUBc4oBCY2c+BnwPUrHk696TtK/lgy4gzKse+6YhyUZjykag05CMrK4uPP/6YvXv3snPnThYtWsSWLVt47bXXeOaZZ3jsscfIysoiNzeXrKwsAP7yl79wwQUX0LdvX5YvX85PfvITJkyYQLlyxXN2P5SexWaWDbQFngX+4O5/D+a/B9zh7gsOtf1ZDRt5ub5jSjzOsmJ42j5GLy2TPYaKnXKRSPlIVBrykf1AT371q1/x/PPPU6FCBb777jt27NhBpUqVqFSpEieeeCIQu0DcsGFD1q5dS4sWLXj77bc588wzAWjYsCFz586lVq1aSe/XzBa4e9siF7p7yh9ANlATeBroFzd/FVD7cNs3btzYZb/Zs2eHHUKpoVwkUj4SlcZ8zJ4923v27HnA/KpVqxY879atm0+cONHd3VesWOG1a9f2vLy8I9oPMN8P8pka9u2j04EBFtMe2O66PiAikmD06NGMHz+eVq1a0a9fPyZNmoSZFdvrh33MOAPoAawFdgMDww1HRCS1MjMzi+yjnJOTU/C8efPmzJkzp8RiCKUQuHuDuMkbw4hBRERiwj41JCIiIVMhEBGJOBUCEZGIUyEQEYk4FQIRkYhTIRARiTgVAhGRiFMhEJFS5bvvvuP888+nVatWtGjRgt/+9rdAbDicu+++m8aNG9OsWTP++Mc/AjBt2jTS09PJyMigbdu2/P3vfw8z/DIplB+UmdnNwPXAyUA14Mtg0VR3vy+MmESkdKhUqRKzZs2iWrVq7N27l06dOtG9e3dWrlzJ+vXr+eyzzyhXrhwbN24E4KKLLuLSSy/FzFiyZAl9+/bls88+C/ldlC1hN6apD9zm7r0Os34CNaZJVBqabZQWykWispSP/IYtZlbQpnHv3r3s3bsXM+PJJ5/kpZdeKhh6OX/kzfiWjrt27SrWMXiiIuWnhgo1pjkv1fsXkdIvNzeXjIwMatWqRdeuXWnXrh2ff/45r7zyCm3btqV79+6sWbOmYP3XX3+dpk2b0rNnTyZMmBBi5GVT2P0IWgJTiDWk+ZrY0cHyg2wT35imzT2PjU9NsGXAGZXh2z1hR1E6KBeJylI+0upWP2BeTk4Ov/nNb7j55pu54YYbGDhwIH379uWDDz7gtddeK7hOkG/x4sU899xzjB49ush95OTkJBxBREmXLl0O2o8g7ELwA5Dn7jlm1gMY4+7nHm77Jk2a+KpVq0o4yrIjKyuryNELo0i5SHQ85OPee++latWqPPPMM7z99ts0aNAAd+eUU05h+/btB6x/9tln88knn1CzZs0Dlh0P+Thah2pME+pdQ+6+w91zguczgIpmduB/PRGJjE2bNrFt2zYA9uzZw7vvvkvTpk25/PLLmTVrFgDvv/8+jRs3BmDt2rX5ja349NNP+eGHHzjttNPCCb6MCrUfgZn9CPjW3d3MzidWmLaEGZOIhOubb77h2muvJTc3l7y8PPr27UuvXr3o1KkT11xzDY8++ijVqlXjmWeeAWDKlCk899xzVKxYkcqVK/PKK6/ogvERCrsxzZXA9Wa2D9gDXOVhnKsSkVIjPT2dhQsXHjD/lFNO4a9/PfAOqDvvvJM777wzFaEdt8JuTPNE8BARkZDol8UiIhGnQiAiEnEqBCIiEadCICIScSoEIiIRp0IgIhJxKgQicswO1kNg0KBBtGrVivT0dK688kpycnIKtvnLX/5C8+bNadGiBVdffXVYoQvh9yM4C8gfQrAC0Aw43d3/HUZcInJ0DtZD4NFHH+Xkk08GYNiwYTzxxBPcddddrFmzhj/84Q/MmTOHU089taC3gIQj1H4E7p7fkAYz6w3cqiIgUvYcrIdAfhFwd/bs2VMw9MP48eO58cYbOfXUU4H9vQUkHCkvBPH9CMxsgrs/GizqB0xO5jXUmCZRWWo+UtKUi0SpyEd+Q5nc3FzatGnD2rVrufHGG2nXrh0AAwcOZMaMGTRv3rxgeOjVq1cD0LFjR3JzcxkxYgTdunUr0Tjl4FJ+jcDdhxDrPdAlvwiYWRWgG7HeBCJSBpUvX55FixaxYcMG5s2bx7JlywCYOHEiX3/9Nc2aNeOVV14BYN++faxZs4asrCwmT57M4MGDC0YcldQLe9C5fL2BOYc6LVSoMQ33pO1LVWyl3hmVY9/8RLkoLBX5yMrKOmBegwYNGDt2LD/96U8L5jVu3Jhx48Zx9tlnU65cOZo0acKcOXOA2Kmhl19+maZNm5ZorDk5OUXGG3nunvIHkA3UjJt+Hbg62e0bN27sst/s2bPDDqHUUC4SpSofGzdu9K1bt7q7++7du71Tp04+ffp0X7Nmjbu75+Xl+fDhw3348OHu7v7WW2/5gAED3N1906ZNXq9ePd+8eXOJxxnl/z+A+X6Qz9TQjwjMrDpwIdA/7FhE5OgU1UOgZ8+eXHDBBezYsQN3p1WrVjz55JMAXHLJJbzzzjs0b96c8uXLM2rUKDWTCVHohQDoA7zj7rvCDkREjs7Begjkn/opzMx45JFHeOSRR0o6NElC2P0IcPdJwKQw4hAREf2yWEQk8lQIREQiToVARCTiVAhERCJOhUBEJOJUCEREIk6FQEQk4lQIRI5D69evp0uXLlx77bW0aNGCMWPGALB48WI6dOhAWloavXv3ZseOHQnbrVu3jmrVqvHwww+HEbaEJJRCYGY3m9lKM3vRzDLNbJGZLTez98OIR+R4U6FCBUaPHs2zzz7L3LlzGTt2LCtWrGDw4ME88MADLF26lD59+jBq1KiE7W699Va6d+8eUtQSlrCOCG4AegA3An8CLnX3FsD/CykekeNK7dq1ad26NQAnnXQSzZo146uvvmLVqlV07twZgK5duzJlyv6R39944w0aNmxIixYtQolZwhNqYxrgZWCqu68DcPek+tWpMU0iNWPZL+q5yG8SkzAvO5uFCxfSrl07WrZsyfTp07nssst49dVXWb9+PQC7du3iwQcfZObMmTotFEEWG500xTs1ywbaAv8LVARaACcBY9z9uYNsE9+PoM09j41PTbBlwBmV4ds9YUdROkQ9F2l1qydMb9q0ibvvvpv+/fvTuXNn1q1bx+OPP8727dvp2LEjU6dOZdq0aTz55JM0bdqULl26MGnSJCpXrpzQS+B4kZOTU9BSM2q6dOmywN3bFrUs7EIwIvh7EVAZ+Bjo6e6rD7X9WQ0bebm+Y0o4yrJjeNo+Ri8tDQPJhi/quYg/Iti7dy//8R//Qb9+/Rg2bNgB665evZr+/fszb948LrjggoKjg23btlGuXDnuu+8+fvnLX6Ys9lTIysoiMzMz7DBCYWYHLQRh/4vZAGwOhqDeZWYfAK2AQxaCyhXLs6qIQ+CoysrKIvuazLDDKBWUixh3Z9CgQdSvXz+hCGzcuJFatWqRl5fHyJEjGTJkCAAffvhhwTojRoygWrVqx10RkIML+/bRacAFZlYh6FvcDlgZckwiZd6cOXN4/vnnWbhwIRkZGWRkZDBjxgwmT55M48aNadq0KXXq1GHgwIFhhyqlQKhHBO6+0szeBpYAecAz7r4szJhEjgedOnXC3Ys8FTJ06NBDbjtixIiSC0xKpdLQmGYUMOrga4uISEkK+9SQiIiETIVARCTiVAhERCJOhUBEJOJUCEREIk6FQEQk4lQIREQiToVA5BjlN4Fp1qxZQhOYfA8//DBmxubNm4HY8A8333wzjRo1Ij09nU8//TSMsEUKlFghiGs+M8XMPjaz783strjlZ5rZ7GCd5WZ26J87ipRS+U1gVq5cmdAEBmJFYubMmZx11lkF67/11lusWbOGNWvWMG7cOK6//vqwQhcBSvaXxTcA3YFdQH3g8kLL9wHD3f1TMzsJWGBmM919xeFeWP0IEkV9DP54qc5F9gM9qV27NrVr1wYSm8A0b96cW2+9lYceeojLLrusYJtp06YxYMAAzIz27duzbds2vvnmm4LXEEm1EjkiKNR85hp3/wTYG7+Ou3/j7p8Gz3cSG2yubknEI5Iq8U1gpk+fTt26dWnVqlXCOl999RVnnnlmwXS9evX46quvUh2qSIESOSJw9yFm1g3o4u6bD7e+mTUAzgP+cYh14hvTcE/avuIJ9jhwRuXYN2FJfS6ysrIKnu/Zs4ehQ4cyePBgPvroI+68805GjRpFVlYW3333HXPmzKF69eps3ryZhQsXsm9fLM6tW7eyYMECcnJyij2+nJychBijTvkoWtj9CDCzasAU4BZ333Gw9dx9HDAOYo1potx8pLCoN2OJl+pc5Pc+2Lt3L7169WLIkCEMGzaMpUuXsmXLloIx/Tdv3sxNN93EvHnzaNWqFTVr1iwYFXTXrl1ceumlJXJqKMqNWIqifBQt1E8PM6tIrAi86O5Tk91OjWkSqRnLfmHkIr8JTLNmzQqawKSlpbFx4/4W3A0aNGD+/PnUrFmTSy+9lCeeeIKrrrqKf/zjH1SvXl3XByRUoRUCMzPgz8BKd38krDhEjlV+E5i0tDQyMjIAuP/+++nRo0eR6/fo0YMZM2bQqFEjqlSpwsSJE1MZrsgBSrwQmNmPgPnAyUCemd0CNAfSgf8GlprZomD1X7v7jJKOSaQ45TeBOZTs7OyC52bG2LFjSzgqkeSVWCGIbz4D1Ctilb8DVlL7FxGR5Bzx7aNmdqqZpZdEMCIiknpJFQIzyzKzk82sBrAYmGhmOq8vInIcSPaIoHpwa+cVwER3bwP8V8mFJSIiqZJsIahgZrWBvsCbJRiPiIikWLKF4D7gb8Dn7v6JmTUE1pRcWCIikipJ3TXk7q8Cr8ZNfwH8pKSCEhGR1En2YnFjM3vPzJYF0+lm9r8lG5qIiKRCsqeGxgO/IhhB1N2XAFeVVFAiR+vBBx+kVq1atGzZsmDe4sWL6dChA2lpafTu3ZsdO/YPabVkyRI6dOhAixYtSEtL47vvvgsjbJFQJVsIqrj7vELzjnqIx7imNS8G0z82s1wzu/JoX1MEoFu3brz99tsJ8wYPHswDDzzA0qVL6dOnD6NGjQJg37599O/fn6eeeorly5eTlZVFxYoVwwhbJFTJ/rJ4s5mdAzhA8IH9zTHs9wagu7t/aWblgQeJXYxOihrTJFJjmliDGIBWrVpRo0aNhGWrVq2ic+fOAHTt2pVLLrmE3/3ud7zzzjukp6cX9As47bTTUhu0SCmR7BHBjcDTQFMz+wq4BRhyNDuMb1pjZrcCNxEbgXTjITcUOUotW7Zk+vTpALz66qusX78egNWrV2NmXHLJJbRu3ZqHHnoozDBFQnPYIwIzKwe0dff/MrOqQLmgo9hRiW9aA1QCXgL+E/jxYeJQY5qDUGOa/Q1icnJymDt3Lrt27SqYN2TIEEaOHMntt99Ox44dKVeuHFlZWaxatYp3332Xp556ikqVKjF8+HDKly9PmzZtwnsjxUyNWBIpH0U7bCFw9zwz+yXwF3ffVcz7fwy4091zY6NSHzKOgsY0TZo08ZuuueyQ60dJVlYWfdVsA4jlomXLllStWjWhAcmAAQOA2FHA8uXLyczM5F//+hd79uwp6Cf8ySefkJeXd1w1LlEjlkTKR9GSPTU008xuM7MzzaxG/qMY9t8WeNnMsoErgT+ZWeEm9yLHJL9BTF5eHiNHjmTIkNhZzUsuuYQlS5awe/du9u3bx/vvv0/z5s3DDFUkFMleLL4u+Htj3Dwndq7/qLn72fnPzWwS8Ka7v3EsrynR9rvf/Y4VK1awefNm6tWrx7333ktOTk7B+P9XXHEFAwcOBODUU09l2LBh/PjHP8bM6NGjBz17qvOdRE+yvyw++/BriYTvN7/5TZGH/kOHDi1y/f79+9O/f/8SjkqkdEuqEJjZgKLmu/tzR7PTQk1r8uf97GheS0REjk2yp4bi7+g5EbgI+BQ4qkIgIiKlR7Knhm6Knzaz6sDzJRKRiIik1BG3qgzsBs4tzkBERCQcyV4j+D+C4SWIFY/mxA1LLSIiZVey1wgejnu+D/inu28ogXhERCTFkj011MPd3w8ec9x9g5k9WKKRiYhISiRbCLoWMa97cQYicqSuu+66A3oPrF27lvbt25ORkUHbtm2ZNy82evrWrVvp06cP6enpnH/++SxbtiyssEVKnUMWAjO73syWAk3MbEnc40tgyWG2ze85MMXMPjaz783stkLrTDCzjfmdz0SOxM9+9rMDeg88/fTT/Pa3v2XRokXcd9993HHHHQDcf//9ZGRksGTJEp577rmD/sBMJIoOd0TwEtAbmB78zX+0cffD/RzzBqAHcD1wM4nXGfJNArodQbwiBTp37nxA7wGgoAPZ9u3bqVOnDgArVqzgoosuAqBp06ZkZ2fz7bffpi5YkVLskBeL3X07sB3oB2BmtYj9oKyamVVz93VFbRffcwCY4O6PmtkBg7i4+wdm1uBIg1ZjmkRRa0yT34SmKL/85S+5/fbbue2228jLy+Ojjz4CYg1rpk6dSqdOnZg3bx7//Oc/2bBhA2eccUaqwhYptZK9fbQ38AhQh1gDmfrASqBFUevH9xxw983FEaj6ERxc1PoRxI8n/69//Suh98Brr73GoEGDuPDCC5k9ezZXXHEFo0ePpmPHjjzxxBM0atSIhg0b0qhRIxYuXMjOnUfdWqNM0Pj7iZSPopm7H34ls8XEmse86+7nmVkXoJ+7//wQ22QTa2izOZgeAeS4+8OF1mtAbNTRliTprIaNvFzfMcmuftwbnraP0UuTvRO47Is/IsjOzqZXr14FF3+rVavGzp07MTPcnerVqyc0qwdwd84++2yWLFnCySefnNLYU03j7yeKcj7MbIG7ty1qWbKfHnvdfYuZlTOzcu4+O8zbRytXLM+qQ5weiJqsrCyyr8kMO4xS4bTTTuP9998nMzOTWbNmce65sR/Ab9u2jSpVqnDCCSfwzDPP0Llz5+O+CIgkK9lCsM3MqgEfAi+a2UZiPywTCU2/fv3IyspK6D1w2223MXz4cPbt28eJJ57IuHHjAFi5ciUDBgygfPnyNG/enD//+c8hRy9SeiRbCC4D9hBrWn8NUB24L5kNzexHwHzgZCDPzG4Bmrv7DjObDGQCNc1sA/Bbd9e/UEnK5MmTD5iXlZXFggULDpjfoUMH1qxZk4qwRMqcZEcf3WVm9YFz3f1ZM6sClD/MNg3iJusdZJ1+yQYqIiIlI6lfFpvZ/wCvAU8Hs+oCaikpInIcSHaIiRuBjsAOAHdfA9QqqaBERCR1ki0E37v7D/kTZlaB/cNSi4hIGZZsIXjfzH4NVDazrsR6EfxfyYUlIiKpkmwhuAvYBCwFfgHMAP63pIISEZHUOeRdQ2Z2lruvc/c8YHzwEBGR48jhjggK7gwysyklHIuIiITgcIXA4p43LMlAJHljxoyhZcuWtGjRgsceewyA22+/naZNm5Kenk6fPn3Ytm1byFGKSFlxuELgB3melCSb03Qzs1VmttbM7jrSfUTNsmXLGD9+PPPmzWPx4sW8+eabbNiwga5du7Js2TKWLFlC48aN+cMf/hB2qCJSRhzul8WtzGwHsSODysFzgml398ON2nUDsZaWu4gNXX15/EIzKw+MJdYKcwPwiZlNd/cVh3rRqPYjyH6gJytXrqR9+/ZUqVIFgAsvvJAPP/yQp59+umC99u3b89prr4UVpoiUMYc8InD38u5+sruf5O4Vguf504csAoWa01zj7p8Aewutdj6w1t2/CH6n8DKxcY3kIFq2bMkHH3zAli1b2L17NzNmzGDTpk0J60yYMIHu3dVSWkSSU2KD2CfZnKYusD5uegPQrqgV1Zhmf0OWyy67jA4dOlC5cmXq169Pbm5uwbIXXniBbdu2Ubdu3Ug24FDjkUTKRyLlo2hhdzOxIuYVeS3C3ccB4yDWmCZKjVjy5fccyMzMZNSoUQD8+te/Zvfu3WRmZvLss8+yfPly3nvvvYJTR1ET5cYjRVE+EikfRQv703QDcGbcdD3g68NtFPXGNBs3bqRWrVqsW7eOqVOnMmrUKN5++20efPBB3n///cgWARE5OmEXgk+Ac83sbOAr4Crg6nBDKv1+8pOfsGXLFipWrMjYsWMpX748gwcP5vvvv6dr165A7ILxU089FXKkIlIWpKQQHKY5zS+BvxHrbzDB3ZenIqay7MMPP0yYzsrKYu3atSFFIyJlXYkWgiSb08wgNnaRiIiEINlB50RE5DilQiAiEnEqBCIiEadCICIScSoEIiIRp0IgIhJxKgQiIhGnQlBKrFq1ioyMjILHySefXNB05vHHH6dJkya0aNGCO+64I+RIReR4E8oQE2Z2M3A98AXwA3AO8B1wnbsvCyOmsDVp0oRFixYBkJubS926denTpw+zZ89m2rRpLFmyhEqVKrFx48aQIxWR401YYw3lN6y5Achx9z5m1pRYk5qLDrfx8dSYJruIwfPee+89zjnnHOrXr8/tt9/OXXfdRbx/MKMAAA1lSURBVKVKlQCoVatWqkMUkeNcyk8NFWpYcwPwHoC7fwY0MLMzUh1TafPyyy/Tr18/AFavXs2HH35Iu3btuPDCC/nkk09Cjk5EjjfmfsStiI99p2bZQFtgGHCiuw8zs/OBj4B27r6giG3iG9O0ueex8SmMuOSk1a2eML13716uvPJKJk6cSI0aNRg4cCDnnXceN910E5999hn33XcfL730Emb7Wznk5ORQrVq1VIdeKikXiZSPRFHOR5cuXRa4e9uiloVdCH4AxgDnAUuBpsBgd198qO3PatjIy/UdU9JhpkThU0PTpk1j7NixvPPOOwB069aNu+66q6CZxjnnnMPcuXM5/fTTC7ZRs439lItEykeiKOfDzA5aCELtR+DuO4CBABb7ivtl8Dik47kxzeTJkwtOCwFcfvnlzJo1i8zMTFavXs0PP/xAzZo1Q4xQRI43od4+amanmNkJweRg4IOgOETS7t27mTlzJldccUXBvOuuu44vvviCli1bctVVV/Hss88mnBYSETlWYXcoawY8Z2a5wApgUMjxhKpKlSps2bIlYd4JJ5zACy+8EFJEIhIFoRSCuIY1m4Fzw4hBRERi9MtiEZGIUyEQEYk4FQIRkYhTIRARiTgVAhGRiFMhEBGJOBWCYtagQQPS0tLIyMigbdvEX3M//PDDmBmbN28OKToRkQOF3Y+gKbExhgBygOsPN85QWTB79uwDhoFYv349M2fO5KyzzgopKhGRooV1RHAD0APoCFzo7unA74BxIcVT4m699VYeeughDQ8hIqVOyo8ICvUjmODuHwWL5gL1knmN0taYJn4EUTPj4osvxsz4xS9+wc9//nOmT59O3bp1adWqVYhRiogULeWFwN2HmFk3oIu7x58sHwS8lep4itucOXOoU6cOGzdupGvXrjRt2pTf//73BcNKi4iUNqH2I8gvBGbWBfgT0Mndtxxkm1LbmKZwc5l8kyZNoly5crz++usFrSY3bdpEzZo1efLJJ6lRo0ax7D/KzTYKUy4SKR+JopyPQzWmwd1T/gCygZrB83Tgc6Bxsts3btzYS6OcnBzfsWNHwfMOHTr4W2+9lbBO/fr1fdOmTcW639mzZxfr65VlykUi5SNRlPMBzPeDfKaGOgy1mZ0FTAX+291XhxlLcfj222/p06cPAPv27ePqq6+mW7duIUclInJoYfcjuAc4DfhTcDfNPj/YoUsZ0LBhQxYvPvTdr9nZ2akJRkQkSWH3IxgcPEREJCT6ZbGISMSpEIiIRJwKgYhIxKkQiIhEnAqBiEjEqRCIiEScCoGISMSpEATWr19Ply5daNasGS1atGDMmDEALF68mA4dOpCWlkbv3r3ZsWNHyJGKiBSvUAqBmd1sZivNbIqZfWxm35vZbWHEkq9ChQqMHj2alStXMnfuXMaOHcuKFSsYPHgwDzzwAEuXLqVPnz6MGjUqzDBFRIpd2I1prgduBh4OKY4CtWvXpnXr1gCcdNJJNGvWjK+++opVq1bRuXNnALp27cqUKVPCDFNEpNiVhsY0j5pZz8NslqA4G9PEN5UpmJedzcKFC2nXrh0tW7Zk+vTpXHbZZbz66qusX7++WPYrIlJalJZ+BCOAHHc/6JFBSfUjKNxLYM+ePQwdOpT+/fvTuXNn1q1bx+OPP8727dvp2LEjU6dOZdq0acWy7+IS5THWC1MuEikfiaKcj0P1Iwh79NGkufs4gp7GZzVs5KOXFk/o2ddkFjzfu3cvvXr1YsiQIQwbNqxg/oABAwBYvXo1y5cvJzMzk9IkKyur1MUUFuUikfKRSPkoWpkpBPEqVyzPqiJO6RwLd2fQoEE0a9YsoQhs3LiRWrVqkZeXx8iRIxkyZEix7ldEJGy6fTQwZ84cnn/+eWbNmkVGRgYZGRnMmDGDyZMn07hxY5o2bUqdOnUYOHBg2KGKiBSrsDuU/QiYD5wM5JnZLUBzd0/5zfqdOnXiYNdLhg4dmuJoRERSJ+zGNAD1wohBRERidGpIRCTiVAhERCJOhUBEJOJUCEREIk6FQEQk4lQIREQiToVARCTiIl0IDtaMZsSIEdStWzfhF8YiIserUH5QZmY3E+tF8FkQw1nB34fdfWKq4shvRtO6dWt27txJmzZt6Nq1KwC33nort90Waq8cEZGUCGuIiRuA7kA/oLq79zaz04FVZvaiu/9wqI2Lox9B9gM9qV27NrVr1wYSm9GIiERJyk8NFWpM48BJZmZANeDfwL5UxwSJzWgAnnjiCdLT07nuuuvYunVrGCGJiKREqI1pgO+JFYSmwEnAT929yK/6xd2YJr4hTeFmNP/+97+pXr06ZsaECRPYsmULd9555zHtryRFudlGYcpFIuUjUZTzcajGNGEXgkygIzAMOAeYCbQ63OijZzVs5OX6jjmmGPJbVOY3o7nkkksS+hAUrJedTa9evVi2bNkx7a8kqdnGfspFIuUjUZTzYWaltkPZQOABj1WjtWb2JbGjg3mH2qi4GtMcrBnNN998U3Dt4PXXX6dly5bHvC8RkdIq7EKwDrgI+NDMzgCaAF+kauf5zWjS0tLIyMgA4P7772fy5MksWrQIM6NBgwY8/fTTqQpJRCTlwi4EvwMmmdlSwIA78xvap8LBmtH06NEjVSGIiISuNDSmuTiMGEREJCbSvywWEREVAhGRyFMhEBGJOBUCEZGIUyEQEYk4FQIRkYhTIRARiTgVAhGRiFMhEBGJOBUCEZGIUyEQEYm4UPoRHCsz2wmsCjuOUqQmkLLB+ko55SKR8pEoyvmo7+6nF7Ug7NFHj9aqgzVYiCIzm698xCgXiZSPRMpH0XRqSEQk4lQIREQirqwWgnFhB1DKKB/7KReJlI9EykcRyuTFYhERKT5l9YhARESKiQqBiEjElalCYGbdzGyVma01s7vCjicVzGyCmW00s2Vx82qY2UwzWxP8PTWYb2b2xyA/S8ysdXiRlwwzO9PMZpvZSjNbbmZDg/mRzImZnWhm88xscZCPe4P5Z5vZP4J8vGJmJwTzKwXTa4PlDcKMvySYWXkzW2hmbwbTkc1FsspMITCz8sBYoDvQHOhnZs3DjSolJgHdCs27C3jP3c8F3gumIZabc4PHz4EnUxRjKu0Dhrt7M6A9cGPw/0FUc/I98J/u3grIALqZWXvgQeDRIB9bgUHB+oOAre7eCHg0WO94MxRYGTcd5Vwkx93LxAPoAPwtbvpXwK/CjitF770BsCxuehVQO3hem9gP7ACeBvoVtd7x+gCmAV2VEweoAnwKtCP269kKwfyCfzvA34AOwfMKwXoWduzFmIN6xL4I/CfwJmBRzcWRPMrMEQFQF1gfN70hmBdFZ7j7NwDB31rB/EjlKDiUPw/4BxHOSXAqZBGwEZgJfA5sc/d9wSrx77kgH8Hy7cBpqY24RD0G3AHkBdOnEd1cJK0sFQIrYp7ufU0UmRyZWTVgCnCLu+841KpFzDuucuLuue6eQezb8PlAs6JWC/4et/kws17ARndfED+7iFWP+1wcqbJUCDYAZ8ZN1wO+DimWsH1rZrUBgr8bg/mRyJGZVSRWBF5096nB7EjnBMDdtwFZxK6dnGJm+WOJxb/ngnwEy6sD/05tpCWmI3CpmWUDLxM7PfQY0czFESlLheAT4NzgDoATgKuA6SHHFJbpwLXB82uJnSfPnz8guFOmPbA9/3TJ8cLMDPgzsNLdH4lbFMmcmNnpZnZK8Lwy8F/ELpTOBq4MViucj/w8XQnM8uAkeVnn7r9y93ru3oDY58Msd7+GCObiiIV9keJIHkAPYDWxc6B3hx1Pit7zZOAbYC+xbzCDiJ3HfA9YE/ytEaxrxO6s+hxYCrQNO/4SyEcnYofvS4BFwaNHVHMCpAMLg3wsA+4J5jcE5gFrgVeBSsH8E4PptcHyhmG/hxLKSybwpnKR3ENDTIiIRFxZOjUkIiIlQIVARCTiVAhERCJOhUBEJOJUCEREIq6sNq8XKXZmlkvsFtN8l7t7dkjhiKSMbh8VCZhZjrtXS+H+Kvj+MXBEQqNTQyJJMrPaZvaBmS0ys2VmdkEwv5uZfRr0BHgvmFfDzN4IeiDMNbP0YP4IMxtnZu8AzwUDxo0ys0+CdX8R4luUiNKpIZH9KgejeAJ86e59Ci2/mtgQxr8P+mNUMbPTgfFAZ3f/0sxqBOveCyx098vN7D+B54j1CwBoA3Ry9z1m9nNiw1782MwqAXPM7B13/7Ik36hIPBUCkf32eGwUz4P5BJgQDHr3hrsvMrNM4IP8D253zx+0rBPwk2DeLDM7zcyqB8umu/ue4PnFQLqZ5Y+FU51YEx0VAkkZFQKRJLn7B2bWGegJPG9mo4BtFD108aGGON5VaL2b3P1vxRqsyBHQNQKRJJlZfWLj3Y8nNgJqa+Bj4EIzOztYJ//U0AfANcG8TGCzF9034W/A9cFRBmbW2MyqlugbESlERwQiycsEbjezvUAOMMDdNwXn+aeaWTlifRC6AiOAiWa2BNjN/uGOC3uGWCvST4MhtjcBl5fkmxApTLePiohEnE4NiYhEnAqBiEjEqRCIiEScCoGISMSpEIiIRJwKgYhIxKkQiIhE3P8HUoXGIAQgQqIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from xgboost import XGBRegressor\n",
    "from xgboost import plot_importance\n",
    "from sklearn.datasets import load_boston\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "X,y = load_boston(return_X_y=True) # 波士顿房价\n",
    "x_train, x_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "model = XGBRegressor(max_depth=5, \n",
    "                         learning_rate=0.1, \n",
    "                         n_estimators=160, \n",
    "                         silent=True, \n",
    "                         objective='reg:gamma')\n",
    "model.fit(x_train, y_train)\n",
    "\n",
    "# 对测试集进行预测\n",
    "y_pred = model.predict(x_test)\n",
    "mse_predict = mean_squared_error(y_test, y_pred)\n",
    "print(mse_predict)\n",
    "\n",
    "# 显示重要特征\n",
    "plot_importance(model)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 调参函数\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "def modify_params(parameters):\n",
    "    #param_grid = {param : param_list}\n",
    "    kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=7)\n",
    "    model = XGBClassifier()\n",
    "    grid_search = GridSearchCV(model, parameters, scoring=\"neg_mean_squared_error\", n_jobs=-1, cv=kfold)\n",
    "    grid_result = grid_search.fit(x_train,y_train)\n",
    "    means = grid_result.cv_results_['mean_test_score']\n",
    "    stds = grid_result.cv_results_['std_test_score']\n",
    "    params = grid_result.cv_results_['params']\n",
    "    for mean, stdev, param in zip(means, stds, params):\n",
    "        print(\"mean:%f ,std:%f with: %r\" % (mean, stdev, param))\n",
    "    print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean:-0.044643 ,std:0.060176 with: {'learning_rate': 0.0001}\n",
      "mean:-0.044643 ,std:0.060176 with: {'learning_rate': 0.001}\n",
      "mean:-0.044643 ,std:0.060176 with: {'learning_rate': 0.01}\n",
      "mean:-0.035714 ,std:0.043630 with: {'learning_rate': 0.1}\n",
      "mean:-0.035714 ,std:0.043630 with: {'learning_rate': 0.2}\n",
      "mean:-0.044643 ,std:0.060176 with: {'learning_rate': 0.3}\n",
      "Best: -0.035714 using {'learning_rate': 0.1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hz/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:813: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "# 选择学习率\n",
    "learning_rate_param={\"learning_rate\" : [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3]}\n",
    "modify_params(learning_rate_param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"532pt\" height=\"392pt\"\n",
       " viewBox=\"0.00 0.00 532.04 392.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 388)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-388 528.0352,-388 528.0352,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"214.1424\" cy=\"-366\" rx=\"66.8882\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"214.1424\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">f2&lt;5.05000019</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"131.1424\" cy=\"-279\" rx=\"62.2891\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"131.1424\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">f2&lt;2.5999999</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;1 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"#0000ff\" d=\"M197.3462,-348.3943C185.196,-335.6586 168.6041,-318.2671 155.0703,-304.081\"/>\n",
       "<polygon fill=\"#0000ff\" stroke=\"#0000ff\" points=\"157.349,-301.3991 147.9138,-296.5796 152.2841,-306.231 157.349,-301.3991\"/>\n",
       "<text text-anchor=\"middle\" x=\"213.1424\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">yes, missing</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>2</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"297.1424\" cy=\"-279\" rx=\"85.2851\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"297.1424\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">leaf=&#45;0.0628940761</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;2 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>0&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"#ff0000\" d=\"M233.4076,-348.3808C239.5107,-342.6327 246.2099,-336.1464 252.1424,-330 259.9727,-321.8876 268.2221,-312.7318 275.4887,-304.4395\"/>\n",
       "<polygon fill=\"#ff0000\" stroke=\"#ff0000\" points=\"278.1947,-306.6614 282.1087,-296.816 272.9093,-302.0717 278.1947,-306.6614\"/>\n",
       "<text text-anchor=\"middle\" x=\"272.6424\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">no</text>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>3</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"85.1424\" cy=\"-192\" rx=\"85.2851\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"85.1424\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">leaf=&#45;0.0620006286</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;3 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>1&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"#0000ff\" d=\"M121.6112,-260.9735C115.1967,-248.8418 106.6243,-232.6287 99.4024,-218.9698\"/>\n",
       "<polygon fill=\"#0000ff\" stroke=\"#0000ff\" points=\"102.4299,-217.2077 94.6615,-210.0034 96.2416,-220.4797 102.4299,-217.2077\"/>\n",
       "<text text-anchor=\"middle\" x=\"146.1424\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">yes, missing</text>\n",
       "</g>\n",
       "<!-- 4 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>4</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"251.1424\" cy=\"-192\" rx=\"62.2891\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"251.1424\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">f3&lt;1.6500001</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;4 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>1&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"#ff0000\" d=\"M156.6662,-262.5112C165.75,-256.5049 175.9915,-249.5737 185.1424,-243 197.3506,-234.2301 210.5099,-224.187 221.8418,-215.3451\"/>\n",
       "<polygon fill=\"#ff0000\" stroke=\"#ff0000\" points=\"224.0243,-218.0815 229.7305,-209.1545 219.7029,-212.5746 224.0243,-218.0815\"/>\n",
       "<text text-anchor=\"middle\" x=\"211.6424\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">no</text>\n",
       "</g>\n",
       "<!-- 5 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>5</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"188.1424\" cy=\"-105\" rx=\"81.7856\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"188.1424\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">leaf=0.0985003486</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;5 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>4&#45;&gt;5</title>\n",
       "<path fill=\"none\" stroke=\"#0000ff\" d=\"M236.9723,-174.2152C232.4767,-168.4583 227.5348,-162.0085 223.1424,-156 217.3714,-148.1056 211.2774,-139.3744 205.8205,-131.4005\"/>\n",
       "<polygon fill=\"#0000ff\" stroke=\"#0000ff\" points=\"208.7097,-129.425 200.1945,-123.1208 202.9198,-133.3591 208.7097,-129.425\"/>\n",
       "<text text-anchor=\"middle\" x=\"258.1424\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">yes, missing</text>\n",
       "</g>\n",
       "<!-- 6 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>6</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"350.1424\" cy=\"-105\" rx=\"62.2891\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"350.1424\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">f1&lt;2.9000001</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;6 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>4&#45;&gt;6</title>\n",
       "<path fill=\"none\" stroke=\"#ff0000\" d=\"M271.299,-174.8736C278.2478,-168.9296 286.0622,-162.2007 293.1424,-156 303.0539,-147.3199 313.8446,-137.7073 323.3443,-129.1889\"/>\n",
       "<polygon fill=\"#ff0000\" stroke=\"#ff0000\" points=\"325.8731,-131.6221 330.9727,-122.3353 321.1948,-126.415 325.8731,-131.6221\"/>\n",
       "<text text-anchor=\"middle\" x=\"316.6424\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">no</text>\n",
       "</g>\n",
       "<!-- 7 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>7</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"257.1424\" cy=\"-18\" rx=\"85.2851\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"257.1424\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">leaf=&#45;0.0492660552</text>\n",
       "</g>\n",
       "<!-- 6&#45;&gt;7 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>6&#45;&gt;7</title>\n",
       "<path fill=\"none\" stroke=\"#0000ff\" d=\"M331.3225,-87.3943C317.5825,-74.5407 298.7729,-56.9446 283.5328,-42.6878\"/>\n",
       "<polygon fill=\"#0000ff\" stroke=\"#0000ff\" points=\"285.6282,-39.8552 275.9344,-35.5796 280.8461,-44.9671 285.6282,-39.8552\"/>\n",
       "<text text-anchor=\"middle\" x=\"345.1424\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">yes, missing</text>\n",
       "</g>\n",
       "<!-- 8 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>8</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"442.1424\" cy=\"-18\" rx=\"81.7856\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"442.1424\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">leaf=0.0420712531</text>\n",
       "</g>\n",
       "<!-- 6&#45;&gt;8 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>6&#45;&gt;8</title>\n",
       "<path fill=\"none\" stroke=\"#ff0000\" d=\"M368.76,-87.3943C382.3523,-74.5407 400.9596,-56.9446 416.0358,-42.6878\"/>\n",
       "<polygon fill=\"#ff0000\" stroke=\"#ff0000\" points=\"418.6916,-44.9935 423.5526,-35.5796 413.882,-39.9074 418.6916,-44.9935\"/>\n",
       "<text text-anchor=\"middle\" x=\"409.6424\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">no</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x1bfd8863c8>"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 绘制决策树\n",
    "xgb.to_graphviz(model, num_trees=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
